{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'  # default is ‘last_expr'\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/Users/siyuyang/Source/repos/GitHub_MSFT/CameraTraps')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from collections import Counter, defaultdict\n",
    "from random import sample\n",
    "import math\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from data_management.megadb.schema import sequences_schema_check\n",
    "from data_management.annotations.add_bounding_boxes_to_megadb import *\n",
    "from data_management.megadb.converters.cct_to_megadb import make_cct_embedded, process_sequences, write_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# annotations\n",
    "\n",
    "anno_batch_10 = '/Users/siyuyang/Source/temp_data/CameraTrap/annotations/201910/batch10'\n",
    "anno_batch_11 = '/Users/siyuyang/Source/temp_data/CameraTrap/annotations/201910/MsWLB_imerit_batch_11_18sep2019'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For some (especially the first few) datasets, the sections \"Make into MegaDB format\" and \"Add bounding boxes\" are run after loading the database and spot checking."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## awc_190430"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/awc_190430_20190816_embedded.json') as f:\n",
    "    awc_embedded = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "159273"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(awc_embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'width': 2048,\n",
       " 'file_name': 'WetTropics/2018/Spurgeon Bettong/QWT-S-TSCAM087 (WF269-269)/IMG_0805.JPG',\n",
       " 'image_id': '102328',\n",
       " 'dataset': 'awc_190430',\n",
       " 'height': 1536,\n",
       " 'location': 'QWT-S-TSCAM087 (WF269-269)',\n",
       " 'annotations': {'species': ['empty']}}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "awc_embedded[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "193"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "awc_locs = Counter()\n",
    "\n",
    "for i in awc_embedded:\n",
    "    awc_locs.update([i['location']])\n",
    "    \n",
    "len(awc_locs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some species arrays had two \"empty\" labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in awc_embedded:\n",
    "    if 'annotations' in e:\n",
    "        e['annotations']['species'] = list(set(e['annotations']['species']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bellevue_190602"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/bellevue_190602_20190812_embedded.json') as f:\n",
    "    bellevue_embedded = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1848"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bellevue_embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'image_id': 'd3695634-a9a9-11e9-a392-5cf370671a19',\n",
       " 'file_name': '2019.01.20/newcam/empty/DSCF0057.JPG',\n",
       " 'annotations': {'species': ['empty']},\n",
       " 'dataset': 'bellevue_190602'}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bellevue_embedded[100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## idfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/idfg_20190807_embedded.json') as f:\n",
    "    idfg_embedded = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "693870"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(idfg_embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'image_id': 'Beaverhead_elk~AM206~Trip 1~100RECNX~2016-03-04 15-45-00 T',\n",
       "  'file_name': 'Beaverhead_elk/AM206/Trip 1/100RECNX/2016-03-04 15-45-00 T.JPG',\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['empty']},\n",
       "  'datetime': '04-Mar-2016 15:45:00',\n",
       "  'location': 'Beaverhead_elk+AM206'}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(idfg_embedded, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_species = []\n",
    "no_annotation = []\n",
    "for e in idfg_embedded:\n",
    "    if 'annotations' not in e:\n",
    "        no_annotation.append(e)\n",
    "        continue\n",
    "    if 'species' not in e['annotations']:\n",
    "        no_species.append(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6224"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(no_annotation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'height': 1536,\n",
       " 'file_name': 'CrowCreek_moose/805/100RECNX/IMG_4593.JPG',\n",
       " 'dataset': 'idfg',\n",
       " 'width': 2048,\n",
       " 'image_id': 'CrowCreek_moose~805~100RECNX~IMG_4593',\n",
       " 'location': 'CrowCreek_moose+805'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_annotation[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8776"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(no_species)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'height': 1536,\n",
       " 'file_name': 'Focal_lion/53/101RECNX/IMG_0510.JPG',\n",
       " 'dataset': 'idfg',\n",
       " 'annotations': {'bbox': [{'category': 'animal',\n",
       "    'bbox_rel': [0.296, 0.601, 0.208, 0.376],\n",
       "    'bbox_abs': [607.5616438356172,\n",
       "     923.1780821917747,\n",
       "     426.08219178082305,\n",
       "     578.6301369862963]}]},\n",
       " 'width': 2048,\n",
       " 'image_id': 'Focal_lion~53~101RECNX~IMG_0510',\n",
       " 'location': 'Focal_lion+53'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_species[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in idfg_embedded:\n",
    "    if 'annotations' not in e:\n",
    "        e['annotations'] = {\n",
    "            'species': ['__label_unavailable']\n",
    "        }\n",
    "        continue\n",
    "    if 'species' not in e['annotations']:\n",
    "        if 'bbox' in e['annotations']:\n",
    "            bbox = e['annotations']['bbox']\n",
    "            if len(bbox) == 0:\n",
    "                e['annotations']['species'] = ['empty']\n",
    "            else:\n",
    "                if bbox[0]['category'] == 'animal':\n",
    "                    e['annotations']['species'] = ['unidentified']\n",
    "                elif bbox[0]['category'] == 'person':\n",
    "                    e['annotations']['species'] = ['human']\n",
    "                else:\n",
    "                    e['annotations']['species'] = ['__label_unavailable']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some images from Trip 2 of a location which we got vehicle annotations for were not in the CCT\n",
    "# it seems, so adding them here manually:\n",
    "\n",
    "idfg_missing = ['Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-12 15-01-50 M 5_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-02-19 09-31-54 M 3_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-02-19 09-31-55 M 4_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-14 12-40-00 T.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-12 15-01-47 M 2_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-02-19 09-31-53 M 1_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-02-19 09-31-56 M 5_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-12 15-01-52 M 2_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-12 15-01-48 M 3_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-12 15-01-51 M 1_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-14 12-40-00 M 3_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-12 15-01-46 M 1_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-12 15-01-49 M 4_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-14 12-39-59 M 2_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-02-19 09-31-53 M 2_5.JPG',\n",
    " 'Beaverhead_elk~AM25~Trip 2~100RECNX~2016-03-14 12-39-58 M 1_5.JPG']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "idfg_missing_items = []\n",
    "for i in idfg_missing:\n",
    "    item = {\n",
    "        'image_id': i.split('.JPG')[0],\n",
    "      'file_name': i.replace('~', '/'),\n",
    "      'dataset': 'idfg',\n",
    "      'annotations': {'species': ['__label_unavailable']},\n",
    "      'location': 'Beaverhead_elk+AM25'\n",
    "    }\n",
    "    idfg_missing_items.append(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "693870"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "693886"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(idfg_embedded)\n",
    "idfg_embedded.extend(idfg_missing_items)\n",
    "len(idfg_embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'height': 1536,\n",
       "  'image_id': 'Beaverhead_elk~AM196~Trip 1~100RECNX~2016-01-06 10-35-00 T',\n",
       "  'file_name': 'Beaverhead_elk/AM196/Trip 1/100RECNX/2016-01-06 10-35-00 T.JPG',\n",
       "  'width': 2048,\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['deer'],\n",
       "   'bbox': [{'category': 'animal',\n",
       "     'bbox_rel': [0.0198, 0.602, 0.0305, 0.0491],\n",
       "     'bbox_abs': [40.62119711261901,\n",
       "      924.9134111796326,\n",
       "      62.49414940402893,\n",
       "      75.43936606629273]},\n",
       "    {'category': 'animal',\n",
       "     'bbox_rel': [0.53, 0.432, 0.0854, 0.055],\n",
       "     'bbox_abs': [1087.0,\n",
       "      664.5103759765555,\n",
       "      174.9486083984384,\n",
       "      84.48962402343781]}]},\n",
       "  'datetime': '06-Jan-2016 10:35:00',\n",
       "  'location': 'Beaverhead_elk+AM196'},\n",
       " {'image_id': 'Beaverhead_elk~AM37~Trip 1~100RECNX~2016-01-31 12-20-00 T',\n",
       "  'file_name': 'Beaverhead_elk/AM37/Trip 1/100RECNX/2016-01-31 12-20-00 T.JPG',\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['empty']},\n",
       "  'datetime': '31-Jan-2016 12:20:00',\n",
       "  'location': 'Beaverhead_elk+AM37'},\n",
       " {'image_id': 'ClearCreek_mustelids~Summer2015~FS-027~RCNX3713',\n",
       "  'file_name': 'ClearCreek_mustelids/Summer2015/FS-027/RCNX3713.JPG',\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['empty']},\n",
       "  'datetime': '8/16/15 5:49:09',\n",
       "  'location': 'ClearCreek_mustelids+FS-027'},\n",
       " {'height': 1536,\n",
       "  'file_name': 'Focal_lion/49/100RECNX/IMG_2772.JPG',\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'bbox': [{'category': 'animal',\n",
       "     'bbox_rel': [0.593, 0.117, 0.169, 0.453],\n",
       "     'bbox_abs': [1215.0, 180.0, 347.0, 696.0]},\n",
       "    {'category': 'animal',\n",
       "     'bbox_rel': [0.733, 0.0214, 0.0757, 0.144],\n",
       "     'bbox_abs': [1501.8082275390668,\n",
       "      33.0,\n",
       "      155.17808219177985,\n",
       "      222.12329101562878]}],\n",
       "   'species': ['unidentified']},\n",
       "  'width': 2048,\n",
       "  'image_id': 'Focal_lion~49~100RECNX~IMG_2772',\n",
       "  'location': 'Focal_lion+49'},\n",
       " {'image_id': 'St.Joe_elk~AM160~Trip 1~100RECNX~RCNX1094',\n",
       "  'file_name': 'St.Joe_elk/AM160/Trip 1/100RECNX/RCNX1094.JPG',\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['empty']},\n",
       "  'datetime': '17-Jan-2016 14:00:00',\n",
       "  'location': 'St_Joe_elk+AM160'},\n",
       " {'image_id': 'St.Joe_elk~AM148~Trip 1~100RECNX~RCNX1954',\n",
       "  'file_name': 'St.Joe_elk/AM148/Trip 1/100RECNX/RCNX1954.JPG',\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['empty']},\n",
       "  'datetime': '19-Apr-2016 01:56:27',\n",
       "  'location': 'St_Joe_elk+AM148'},\n",
       " {'height': 1536,\n",
       "  'image_id': 'Beaverhead_elk~AM196~Trip 1~100RECNX~2016-02-02 13-15-57 M 4_5',\n",
       "  'file_name': 'Beaverhead_elk/AM196/Trip 1/100RECNX/2016-02-02 13-15-57 M 4_5.JPG',\n",
       "  'width': 2048,\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['deer'],\n",
       "   'bbox': [{'category': 'animal',\n",
       "     'bbox_rel': [0.687, 0.489, 0.153, 0.23],\n",
       "     'bbox_abs': [1408.6602822157108,\n",
       "      752.633484118441,\n",
       "      314.5333963480064,\n",
       "      353.8500708914995]},\n",
       "    {'category': 'animal',\n",
       "     'bbox_rel': [0.49, 0.455, 0.315, 0.291],\n",
       "     'bbox_abs': [1004.2602011968512,\n",
       "      699.8368068743116,\n",
       "      645.9167960717926,\n",
       "      447.08675623752185]}]},\n",
       "  'datetime': '02-Feb-2016 13:15:57',\n",
       "  'location': 'Beaverhead_elk+AM196'},\n",
       " {'image_id': 'St.Joe_elk~AM109~Trip 1~100RECNX~RCNX0877',\n",
       "  'file_name': 'St.Joe_elk/AM109/Trip 1/100RECNX/RCNX0877.JPG',\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['elk']},\n",
       "  'datetime': '23-Apr-2016 16:11:33',\n",
       "  'location': 'St_Joe_elk+AM109'},\n",
       " {'image_id': 'St.Joe_elk~AM118~Trip 1~100RECNX~RCNX0736',\n",
       "  'file_name': 'St.Joe_elk/AM118/Trip 1/100RECNX/RCNX0736.JPG',\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['elk']},\n",
       "  'datetime': '28-Feb-2016 02:53:57',\n",
       "  'location': 'St_Joe_elk+AM118'},\n",
       " {'image_id': 'Beaverhead_elk~AM206~Trip 1~100RECNX~2016-01-15 07-05-00 T',\n",
       "  'file_name': 'Beaverhead_elk/AM206/Trip 1/100RECNX/2016-01-15 07-05-00 T.JPG',\n",
       "  'dataset': 'idfg',\n",
       "  'annotations': {'species': ['empty']},\n",
       "  'datetime': '15-Jan-2016 07:05:00',\n",
       "  'location': 'Beaverhead_elk+AM206'}]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(idfg_embedded, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nacti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/nacti_20190819_embedded.json') as f:\n",
    "    nacti_embedded = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3382215"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nacti_embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'height': 1536,\n",
       "  'width': 2048,\n",
       "  'annotations': {'species': ['domestic cow']},\n",
       "  'image_id': 'FL-07_06_29_2015_FL-07_0042512',\n",
       "  'location': 'Archbold, FL',\n",
       "  'study': 'FL',\n",
       "  'dataset': 'nacti',\n",
       "  'file_name': 'part1/sub147/FL-07_06_29_2015_FL-07_0042512.jpg'},\n",
       " {'height': 1536,\n",
       "  'width': 2048,\n",
       "  'annotations': {'species': ['red deer']},\n",
       "  'image_id': '2014_Unit70_Ivan053_img0337',\n",
       "  'location': 'San Juan Mntns, Colorado',\n",
       "  'study': 'CPW',\n",
       "  'dataset': 'nacti',\n",
       "  'file_name': 'part0/sub015/2014_Unit70_Ivan053_img0337.jpg'}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(nacti_embedded, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "species_counter = Counter()\n",
    "for i in nacti_embedded:\n",
    "    species_counter.update(i['annotations']['species'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nacti_locs = Counter()\n",
    "\n",
    "for i in nacti_embedded:\n",
    "    nacti_locs.update([i['location']])\n",
    "    \n",
    "len(nacti_locs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'Archbold, FL': 1802622,\n",
       "         'Lebec, California': 780549,\n",
       "         '': 469356,\n",
       "         'San Juan Mntns, Colorado': 329688})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nacti_locs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Counter({'CA': 261600, 'FL': 207756})\n",
    "\n",
    "for i in nacti_embedded:\n",
    "    if i['location'] == '':\n",
    "        i['location'] = i['image_id'].split('-')[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rspb_gola_20190409"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/rspb_gola_20190409_embedded.json') as f:\n",
    "    gola_embedded = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "62835"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gola_embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'location': 'P0440__C11',\n",
       "  'file_name': 'P0440/C11/P0440__C11__2012-05-26__10-16-28(6).JPG',\n",
       "  'datetime': '26/05/2012 10:16',\n",
       "  'image_id': 'P0440__C11__2012-05-26__10-16-28(6)',\n",
       "  'dataset': 'rspb_gola',\n",
       "  'annotations': {'species': ['empty']},\n",
       "  'height': 1536,\n",
       "  'width': 2048}]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(gola_embedded, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add sequence info to the RSPB database using info in the file names. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in gola_embedded:\n",
    "    frame_num = int(e['image_id'].split('(')[1].split(')')[0])\n",
    "    seq_id = e['image_id'].split('(')[0]\n",
    "    e['seq_id'] = seq_id\n",
    "    e['frame_num'] = frame_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in gola_embedded:\n",
    "    if 'species' not in e['annotations']:\n",
    "        e['annotations']['species'] = ['__label_unavailable']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## wcs_20190817"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/wcs_20190817_embedded.json') as f:\n",
    "    wcs_embedded = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1207490"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(wcs_embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'datetime': '2008-02-19 13:09:05.000',\n",
       "  'seq_id': 'ken-007-d0037-1040',\n",
       "  'seq_num_frames': 33,\n",
       "  'frame_num': 2,\n",
       "  'match_level': 1,\n",
       "  'file_name': 'animals/0200/0530.jpg',\n",
       "  'width': 1920,\n",
       "  'image_id': '0fb7c9ee-92d5-11e9-a2bf-000d3a74c7de',\n",
       "  'location': '1256',\n",
       "  'corrupt': False,\n",
       "  'dataset': 'wcs',\n",
       "  'annotations': {'sex': 'unknown',\n",
       "   'species': ['empty'],\n",
       "   'count': -1,\n",
       "   'age': 'unknown'},\n",
       "  'wcs_id': 'ken-007-d0037-i012594',\n",
       "  'country_code': 'ken',\n",
       "  'height': 1080}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(wcs_embedded, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "wcs_locations = Counter()\n",
    "\n",
    "for e in wcs_embedded:\n",
    "    wcs_locations.update([e['location']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2191"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wcs_locations['unknown']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "seqs_unknown = []\n",
    "\n",
    "for e in wcs_embedded:\n",
    "    if e['seq_id'] == 'unknown':\n",
    "        seqs_unknown.append(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "242053"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(seqs_unknown)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'datetime': '2013-07-12 04:07:43.000',\n",
       " 'seq_id': 'unknown',\n",
       " 'seq_num_frames': -1,\n",
       " 'frame_num': -1,\n",
       " 'match_level': 4,\n",
       " 'file_name': 'animals/0560/0614.jpg',\n",
       " 'width': 1920,\n",
       " 'image_id': '9475a0e7-92d5-11e9-9bd6-000d3a74c7de',\n",
       " 'location': '4342',\n",
       " 'corrupt': False,\n",
       " 'dataset': 'wcs',\n",
       " 'annotations': {'sex': 'male',\n",
       "  'species': ['leopardus pardalis'],\n",
       "  'count': 1,\n",
       "  'age': 'unknown'},\n",
       " 'wcs_id': 'gtm-003-d0045-i069662',\n",
       " 'country_code': 'gtm',\n",
       " 'height': 1080}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seqs_unknown[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete the sequence properties for these sequences so that dummy seq ID can be created for them "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in wcs_embedded:\n",
    "    if e['seq_id'] == 'unknown':\n",
    "        del e['seq_id']\n",
    "        del e['seq_num_frames']\n",
    "        del e['frame_num']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "78"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrupt_ims = []\n",
    "for e in wcs_embedded:\n",
    "    if e['corrupt']:\n",
    "        corrupt_ims.append(e)\n",
    "        \n",
    "len(corrupt_ims)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "141"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_jpg = []\n",
    "for e in wcs_embedded:\n",
    "    if not e['file_name'].endswith('.jpg'):\n",
    "        non_jpg.append(e)\n",
    "len(non_jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Added `.bmp`, `.avi`, `.tif` as allowed endings for `file` in the schema"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some datetime are nan - make these into string so the schema check can pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in wcs_embedded:\n",
    "    if type(e['datetime']) is not str:\n",
    "        e['datetime'] = str(e['datetime'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weird_class_label = []\n",
    "for e in wcs_embedded:\n",
    "    if '#ref!' in e['annotations']['species']:\n",
    "        weird_class_label.append(e)\n",
    "len(weird_class_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These appear to be cars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in wcs_embedded:\n",
    "    if '#ref!' in e['annotations']['species']:\n",
    "        e['annotations']['species'] = ['vehicle']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## wcs_private"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/wcs_private_20190819_embedded.json') as f:\n",
    "    wcs_private_embedded = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "162501"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(wcs_private_embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in wcs_private_embedded:\n",
    "    if e['seq_id'] == 'unknown':\n",
    "        del e['seq_id']\n",
    "        del e['seq_num_frames']\n",
    "        del e['frame_num']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in wcs_private_embedded:\n",
    "    if type(e['datetime']) is not str:\n",
    "        e['datetime'] = str(e['datetime'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## wiitigers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/wiitigers_20190815_embedded.json') as f:\n",
    "    wii_embedded = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1125938"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'annotations': {'species': ['human']},\n",
       "  'file_name': 'raw_camtrap/TATR DATA_2014,15,16,17/2018/block_1/73/A/tourist/I__01068.JPG',\n",
       "  'width': 2576,\n",
       "  'dataset': 'wiitigers',\n",
       "  'height': 1984,\n",
       "  'image_id': 'raw_camtrap~TATR DATA_2014,15,16,17~2018~block_1~73~A~tourist~I__01068'}]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(wii_embedded)\n",
    "sample(wii_embedded, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# manually fixed this oen image\n",
    "for i in wii_embedded:\n",
    "    if not i['file_name'].endswith(('.JPG', '.jpg')):\n",
    "        i['file_name'] = i['file_name'].split('.JFG')[0] + '.JPG'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**The following datasets that were annotated in batch 10 did not have their embedded CCT compiled... Some do not yet have a database json.**\n",
    "\n",
    "## trailguard_night_mara_190515"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'trailguard_night_mara_190515'\n",
    "annotation_path = anno_batch_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▎         | 6560/177122 [00:00<00:05, 32792.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 files found in directory at annotation_path\n",
      "Number of annotation entries found: 177122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 177122/177122 [00:05<00:00, 31801.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54 boxes on 63 images were in the annotation file(s). 215715 boxes skipped because they are not for the requested dataset\n",
      "\n",
      "Category counts for the bboxes:\n",
      "animal: 1\n",
      "person: 53\n"
     ]
    }
   ],
   "source": [
    "image_filename_to_bboxes = extract_annotations(annotation_path, dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# checked that this label should be person, not animal\n",
    "\n",
    "image_filename_to_bboxes['tgv3_01003.jpg'][0]['category'] = 'person'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences = []\n",
    "\n",
    "for image_filename, bbox in image_filename_to_bboxes.items():\n",
    "    im_cat = 'person' if len(bbox) > 0 else 'empty'\n",
    "    sequences.append({\n",
    "        'dataset': 'trailguard_night_mara_190515',\n",
    "        'seq_id': 'dummy_' + image_filename.split('.jpg')[0],\n",
    "        'images': [\n",
    "            {\n",
    "                'file': image_filename,  # all flat, no separator /\n",
    "                'class': [im_cat]\n",
    "            }\n",
    "        ]\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset to which the sequences belong to: trailguard_night_mara_190515. The bboxes should also be for this set.\n",
      "63 images updated; 0 images had their bbox overwritten; 0 images not updated\n"
     ]
    }
   ],
   "source": [
    "sequences, images_updated = add_annotations_to_sequences(sequences, image_filename_to_bboxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n"
     ]
    }
   ],
   "source": [
    "sequences_schema_check.sequences_schema_check(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/megadb_batches_9_10_11/trailguard_night_mara_190515_w_batch_10_boxes.json', sequences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## islandconservation_190705"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'islandconservation_190705'\n",
    "annotation_path = anno_batch_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 2901/177122 [00:00<00:12, 14410.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 files found in directory at annotation_path\n",
      "Number of annotation entries found: 177122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 177122/177122 [00:08<00:00, 21980.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10806 boxes on 30000 images were in the annotation file(s). 204963 boxes skipped because they are not for the requested dataset\n",
      "\n",
      "Category counts for the bboxes:\n",
      "animal: 8695\n",
      "person: 2102\n",
      "vehicle: 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "image_filename_to_bboxes = extract_annotations(annotation_path, dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10806"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "9 + 2102 + 8695  # about two-thirds were empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences = []\n",
    "count = 0\n",
    "for image_filename, bbox in image_filename_to_bboxes.items():\n",
    "    im_cat = 'empty' if len(bbox) == 0 else bbox[0]['category']\n",
    "    if im_cat == 'animal':\n",
    "        im_cat = 'unidentified'\n",
    "    sequences.append({\n",
    "        'dataset': 'islandconservation_190705',\n",
    "        'seq_id': 'dummy_' + dataset_name + '_' + str(count),\n",
    "        'images': [\n",
    "            {\n",
    "                'file': image_filename.replace('~', '/'),\n",
    "                'class': [im_cat]\n",
    "            }\n",
    "        ]\n",
    "    })\n",
    "    \n",
    "    count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset to which the sequences belong to: islandconservation_190705. The bboxes should also be for this set.\n",
      "30000 images updated; 0 images had their bbox overwritten; 0 images not updated\n"
     ]
    }
   ],
   "source": [
    "sequences, images_updated = add_annotations_to_sequences(sequences, image_filename_to_bboxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'islandconservation_190705',\n",
       "  'seq_id': 'dummy_islandconservation_190705_1268',\n",
       "  'images': [{'file': 'Ngeruktabel Camera Study/Cam07a/CAM07A083118/Folder 1/RCNX0022.JPG',\n",
       "    'class': ['person'],\n",
       "    'bbox': [{'category': 'person',\n",
       "      'bbox': [0.08307, 0.02226, 0.915, 0.791]}]}]}]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(sequences, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n"
     ]
    }
   ],
   "source": [
    "sequences_schema_check.sequences_schema_check(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/megadb_batches_9_10_11/islandconservation_190705_w_batch_10_boxes.json', sequences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## parkscanada_20190715"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'parkscanada_20190715'\n",
    "annotation_path = anno_batch_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 6775/177122 [00:00<00:05, 33866.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 files found in directory at annotation_path\n",
      "Number of annotation entries found: 177122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 177122/177122 [00:05<00:00, 30678.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 boxes on 22 images were in the annotation file(s). 215747 boxes skipped because they are not for the requested dataset\n",
      "\n",
      "Category counts for the bboxes:\n",
      "person: 5\n",
      "vehicle: 17\n"
     ]
    }
   ],
   "source": [
    "image_filename_to_bboxes = extract_annotations(annotation_path, dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences = []\n",
    "count = 0\n",
    "for image_filename, bbox in image_filename_to_bboxes.items():\n",
    "    im_cat = 'empty' if len(bbox) == 0 else bbox[0]['category']\n",
    "    if im_cat == 'animal':\n",
    "        im_cat = 'unidentified'\n",
    "    sequences.append({\n",
    "        'dataset': 'parkscanada_20190715',\n",
    "        'seq_id': 'dummy_' + 'parkscanada_20190715_cars' + '_' + str(count),\n",
    "        'images': [\n",
    "            {\n",
    "                'file': image_filename.replace('~', '/'),\n",
    "                'class': [im_cat]\n",
    "            }\n",
    "        ]\n",
    "    })\n",
    "    \n",
    "    count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset to which the sequences belong to: parkscanada_20190715. The bboxes should also be for this set.\n",
      "22 images updated; 0 images had their bbox overwritten; 0 images not updated\n"
     ]
    }
   ],
   "source": [
    "sequences, images_updated = add_annotations_to_sequences(sequences, image_filename_to_bboxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n"
     ]
    }
   ],
   "source": [
    "sequences_schema_check.sequences_schema_check(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/megadb_batches_9_10_11/parkscanada_20190715_w_batch_10_boxes.json', sequences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bnf_20190624and0815"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The BNF data is too messy to put into sequences or locations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/Source/temp_data/CameraTrap/batch_9_10_11/BNF/BNF_20190624and0815_20190826.json') as f:\n",
    "    bnf_cct = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading image DB...\n",
      "Number of items from the image DB: 25000\n",
      "Number of images with more than 1 species: 0 (0.0% of image DB)\n",
      "No bbox DB provided.\n"
     ]
    }
   ],
   "source": [
    "bnf_embedded = make_cct_embedded(bnf_cct)  # did not have any bboxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25000"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bnf_embedded)  # only the ones sampled to get bboxes were labeled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in bnf_embedded:\n",
    "    e['image_id'] = e['id']\n",
    "    del e['id']\n",
    "    \n",
    "    if 'annotations' not in e or 'species' not in e['annotations']:\n",
    "        e['annotations'] = {'species': ['__label_unavailable']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'file_name': '20190624cameratraps/images/OUTROP CAMERA TRAPS ALL/Camera traps ALL historical data/CAMERA TRAPS 2011/Camera trap locations 2011/KM 4 x Railway/Km4 Railway 19-11-11/Cam 2011-5/CDY_0071.JPG',\n",
       "  'image_id': 15551,\n",
       "  'annotations': {'species': ['__label_unavailable']}},\n",
       " {'file_name': '20190624cameratraps/images/OUTROP CAMERA TRAPS ALL/Camera traps ALL historical data/CAMERA TRAPS 2010/Best of 2010/Best of Agustus/Km 3 Macaque 12-07-2010 (3).JPG',\n",
       "  'annotations': {'species': ['macaque']},\n",
       "  'image_id': 14204}]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(bnf_embedded, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sulross_kitfox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'sulross_kitfox'\n",
    "annotation_path = anno_batch_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 7041/177122 [00:00<00:04, 35191.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 files found in directory at annotation_path\n",
      "Number of annotation entries found: 177122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 177122/177122 [00:07<00:00, 22213.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72 boxes on 39 images were in the annotation file(s). 215697 boxes skipped because they are not for the requested dataset\n",
      "\n",
      "Category counts for the bboxes:\n",
      "animal: 5\n",
      "person: 29\n",
      "vehicle: 38\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "image_filename_to_bboxes = extract_annotations(annotation_path, dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences = []\n",
    "count = 0\n",
    "for image_filename, bbox in image_filename_to_bboxes.items():\n",
    "    im_cat = 'empty' if len(bbox) == 0 else bbox[0]['category']\n",
    "    if im_cat == 'animal':\n",
    "        im_cat = 'unidentified'\n",
    "    sequences.append({\n",
    "        'dataset': 'sulross_kitfox',\n",
    "        'seq_id': 'dummy_' + 'sulross_kitfox_cars' + '_' + str(count),\n",
    "        'images': [\n",
    "            {\n",
    "                'file': image_filename.replace('~', '/'),\n",
    "                'class': [im_cat]\n",
    "            }\n",
    "        ]\n",
    "    })\n",
    "    \n",
    "    count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset to which the sequences belong to: sulross_kitfox. The bboxes should also be for this set.\n",
      "39 images updated; 0 images had their bbox overwritten; 0 images not updated\n"
     ]
    }
   ],
   "source": [
    "sequences, images_updated = add_annotations_to_sequences(sequences, image_filename_to_bboxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n"
     ]
    }
   ],
   "source": [
    "sequences_schema_check.sequences_schema_check(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/megadb_batches_9_10_11/sulross_kitfox_w_batch_10_boxes.json', sequences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nacti_private"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'nacti_private'\n",
    "annotation_path = anno_batch_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 6149/177122 [00:00<00:05, 30737.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 files found in directory at annotation_path\n",
      "Number of annotation entries found: 177122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 177122/177122 [00:05<00:00, 29838.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1083 boxes on 1000 images were in the annotation file(s). 214686 boxes skipped because they are not for the requested dataset\n",
      "\n",
      "Category counts for the bboxes:\n",
      "animal: 63\n",
      "group: 1\n",
      "person: 842\n",
      "vehicle: 177\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "image_filename_to_bboxes = extract_annotations(annotation_path, dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('FL-11_06_01_2015_FL-11_0000039.jpg',\n",
       " [{'category': 'person', 'bbox': [0.3995, 0.02225, 0.1825, 0.8259]}])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(image_filename_to_bboxes.items())[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences = []\n",
    "count = 0\n",
    "for image_filename, bbox in image_filename_to_bboxes.items():\n",
    "    im_cat = 'empty' if len(bbox) == 0 else bbox[0]['category']\n",
    "    if im_cat == 'animal':\n",
    "        im_cat = 'unidentified'\n",
    "    sequences.append({\n",
    "        'dataset': 'nacti_private',\n",
    "        'seq_id': 'dummy_' + 'nacti_private' + '_' + str(count),\n",
    "        'images': [\n",
    "            {\n",
    "                'file': image_filename.replace('~', '/'),\n",
    "                'class': [im_cat]\n",
    "            }\n",
    "        ]\n",
    "    })\n",
    "    \n",
    "    count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'dataset': 'nacti_private',\n",
       "  'seq_id': 'dummy_nacti_private_258',\n",
       "  'images': [{'file': '2015_Unit097_Ivan076_img1325.jpg',\n",
       "    'class': ['person']}]},\n",
       " {'dataset': 'nacti_private',\n",
       "  'seq_id': 'dummy_nacti_private_753',\n",
       "  'images': [{'file': 'CA-45_10_07_2015_CA-45_0011459.jpg',\n",
       "    'class': ['person']}]}]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(sequences, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset to which the sequences belong to: nacti_private. The bboxes should also be for this set.\n",
      "1000 images updated; 0 images had their bbox overwritten; 0 images not updated\n"
     ]
    }
   ],
   "source": [
    "sequences, images_updated = add_annotations_to_sequences(sequences, image_filename_to_bboxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n"
     ]
    }
   ],
   "source": [
    "sequences_schema_check.sequences_schema_check(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/megadb_batches_9_10_11/nacti_private_w_batch_10_boxes.json', sequences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## caltech"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# correct database with the unidentified_animal problem fixed\n",
    "\n",
    "im_db_path = '/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/databases_caltech_190918/caltech_images_20190919.json'\n",
    "bbox_db_path = '/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/databases_caltech_190918/caltech_bboxes_20190904.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(im_db_path) as f:\n",
    "    caltech_im_db = json.load(f)\n",
    "with open(bbox_db_path) as f:\n",
    "    caltech_bbox_db = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get rid of the train_val and test prefix to the file_name field in the image DB\n",
    "# - these are not in the unzipped version of the images\n",
    "\n",
    "for i in caltech_im_db['images']:\n",
    "    i['file_name'] = i['file_name'].split('/')[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading image DB...\n",
      "Number of items from the image DB: 243100\n",
      "Number of images with more than 1 species: 1569 (0.65% of image DB)\n",
      "Loading bbox DB...\n",
      "Number of images added from bbox DB entries:  1080\n",
      "Number of images amended:  0\n",
      "Number of items in total:  244180\n",
      "Number of images with more than one bounding box: 2571 (1.0529117863870916% of all entries)\n"
     ]
    }
   ],
   "source": [
    "caltech_embedded = make_cct_embedded(image_db=caltech_im_db, bbox_db=caltech_bbox_db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1080"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# In the unzipped version both train_val and test are stored in one folder...\n",
    "caltech_species = Counter()\n",
    "\n",
    "unlabeled = []\n",
    "\n",
    "for e in caltech_embedded: \n",
    "    if 'annotations' in e and 'species' in e['annotations']:\n",
    "        caltech_species.update(e['annotations']['species'])\n",
    "    else:\n",
    "        unlabeled.append(e)\n",
    "len(unlabeled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'date_captured': '2013-02-15 09:32:53',\n",
       " 'file_name': '59e441b7-23d2-11e8-a6a3-ec086b02610b.jpg',\n",
       " 'seq_id': '70136b4c-5567-11e8-8d07-dca9047ef277',\n",
       " 'seq_num_frames': 3,\n",
       " 'rights_holder': 'Justin Brown',\n",
       " 'location': '61',\n",
       " 'frame_num': 1,\n",
       " 'id': '59e441b7-23d2-11e8-a6a3-ec086b02610b',\n",
       " 'annotations': {'bbox': [{'category': 'animal',\n",
       "    'bbox': [0.686, 0.478, 0.0444, 0.0596]}]}}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unlabeled[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'empty': 125745,\n",
       "         'deer': 12196,\n",
       "         'cat': 5297,\n",
       "         'opossum': 16698,\n",
       "         'car': 4717,\n",
       "         'rabbit': 12519,\n",
       "         'dog': 4512,\n",
       "         'raccoon': 11390,\n",
       "         'rodent': 4279,\n",
       "         'coyote': 16601,\n",
       "         'skunk': 1898,\n",
       "         'fox': 2574,\n",
       "         'bird': 10003,\n",
       "         'squirrel': 4450,\n",
       "         'bobcat': 8098,\n",
       "         'lizard': 309,\n",
       "         'badger': 50,\n",
       "         'mountain_lion': 145,\n",
       "         'pig': 2,\n",
       "         'insect': 6,\n",
       "         'bat': 3,\n",
       "         'cow': 3626})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caltech_species"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_species = Counter()\n",
    "for e in caltech_embedded: \n",
    "    if 'species' not in e['annotations']:\n",
    "        e['annotations']['species'] = ['__label_unavailable']\n",
    "        \n",
    "    new_species.update(e['annotations']['species'])\n",
    "\n",
    "    if 'bbox' in e['annotations']:\n",
    "        # this copy already has the coordinates in relative coord\n",
    "        for b in e['annotations']['bbox']:\n",
    "            b['bbox_rel'] = b['bbox']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'empty': 125745,\n",
       "         'deer': 12196,\n",
       "         'cat': 5297,\n",
       "         'opossum': 16698,\n",
       "         'car': 4717,\n",
       "         'rabbit': 12519,\n",
       "         'dog': 4512,\n",
       "         'raccoon': 11390,\n",
       "         'rodent': 4279,\n",
       "         'coyote': 16601,\n",
       "         'skunk': 1898,\n",
       "         'fox': 2574,\n",
       "         'bird': 10003,\n",
       "         'squirrel': 4450,\n",
       "         'bobcat': 8098,\n",
       "         'lizard': 309,\n",
       "         'badger': 50,\n",
       "         'mountain_lion': 145,\n",
       "         'pig': 2,\n",
       "         'insect': 6,\n",
       "         'bat': 3,\n",
       "         'cow': 3626,\n",
       "         '__label_unavailable': 1080})"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_species"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some location fields are str - these were the ones added from the bbox DB.\n",
    "\n",
    "Change `date_captured` to `datetime` which is more standard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in caltech_embedded:\n",
    "    e['location'] = int(e['location'])\n",
    "    e['datetime'] = e['date_captured']\n",
    "    del e['date_captured']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'seq_num_frames': 3,\n",
       "  'seq_id': '6efebedc-5567-11e8-a4be-dca9047ef277',\n",
       "  'height': 1494,\n",
       "  'width': 2048,\n",
       "  'location': 38,\n",
       "  'rights_holder': 'Justin Brown',\n",
       "  'file_name': '59cb3880-23d2-11e8-a6a3-ec086b02610b.jpg',\n",
       "  'id': '59cb3880-23d2-11e8-a6a3-ec086b02610b',\n",
       "  'frame_num': 3,\n",
       "  'annotations': {'species': ['rabbit']},\n",
       "  'datetime': '2011-06-23 09:18:17'},\n",
       " {'seq_num_frames': 3,\n",
       "  'seq_id': '701de9e3-5567-11e8-8874-dca9047ef277',\n",
       "  'height': 584,\n",
       "  'width': 800,\n",
       "  'location': 136,\n",
       "  'rights_holder': 'Justin Brown',\n",
       "  'file_name': '593a4f37-23d2-11e8-a6a3-ec086b02610b.jpg',\n",
       "  'id': '593a4f37-23d2-11e8-a6a3-ec086b02610b',\n",
       "  'frame_num': 3,\n",
       "  'annotations': {'species': ['bird']},\n",
       "  'datetime': '2014-02-10 08:03:47'}]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(caltech_embedded, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset_name is set to caltech. Please make sure this is correct!\n",
      "Making a deep copy of docs...\n",
      "Putting 244180 images into sequences...\n",
      "Number of sequences: 181110\n",
      "Checking the location field...\n",
      "\n",
      "all_img_properties\n",
      "{'datetime', 'id', 'rights_holder', 'bbox', 'file', 'class', 'location', 'frame_num'}\n",
      "\n",
      "img_level_properties\n",
      "{'datetime', 'id', 'bbox', 'file', 'class', 'frame_num'}\n",
      "\n",
      "image-level properties that really should be sequence-level\n",
      "{'rights_holder', 'location'}\n",
      "\n",
      "Finished processing sequences.\n",
      "Example sequence items:\n",
      "\n",
      "{'seq_id': '6f2160eb-5567-11e8-990e-dca9047ef277', 'dataset': 'caltech', 'images': [{'id': '5968c0f9-23d2-11e8-a6a3-ec086b02610b', 'frame_num': 1, 'datetime': '2013-10-04 13:31:53', 'file': '5968c0f9-23d2-11e8-a6a3-ec086b02610b.jpg', 'class': ['empty']}], 'rights_holder': 'Erin Boydston', 'location': 26}\n",
      "\n",
      "[{'seq_id': '6f88cc91-5567-11e8-9568-dca9047ef277', 'dataset': 'caltech', 'images': [{'id': '5a16417d-23d2-11e8-a6a3-ec086b02610b', 'frame_num': 1, 'datetime': '2014-05-16 07:00:00', 'file': '5a16417d-23d2-11e8-a6a3-ec086b02610b.jpg', 'class': ['empty']}], 'rights_holder': 'Erin Boydston', 'location': 57}]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dataset_name = 'caltech'\n",
    "cal_sequences = process_sequences(caltech_embedded, dataset_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some frame_num are not unique in a sequence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "problem_sequences = []\n",
    "\n",
    "for seq in cal_sequences:\n",
    "    if 'images' not in seq:\n",
    "        continue\n",
    "        \n",
    "    # if there are more than one image item, each needs a frame_num\n",
    "    if len(seq['images']) > 1:\n",
    "        frame_num_set = []\n",
    "        for i in seq['images']:\n",
    "            if 'frame_num' not in i:\n",
    "                assert False, 'sequence {} has more than one image but not all images have frame_num'.format(seq['seq_id'])\n",
    "\n",
    "            frame_num_set.append(i['frame_num'])\n",
    "            \n",
    "        if len(set(frame_num_set)) != len(seq['images']):\n",
    "            problem_sequences.append(seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(problem_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/databases_caltech_190918/caltech_problem_seqs.json', problem_sequences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Manually corrected using the timestamp. Some had the same timestamp - chose frame_num randomly in that case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/databases_caltech_190918/caltech_corrected_seqs.json') as f:\n",
    "    corrected_seqs = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrected = {}\n",
    "for s in corrected_seqs:\n",
    "    corrected[s['seq_id']] = s['images']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "for seq in cal_sequences:\n",
    "    if seq['seq_id'] in corrected:\n",
    "        seq['images'] = corrected[seq['seq_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "for seq in cal_sequences:\n",
    "    for im in seq['images']:\n",
    "        im['class'] = list(set(im['class']))  # some dups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('dataset', 'caltech'),\n",
       "             ('seq_id', '6f2160eb-5567-11e8-990e-dca9047ef277'),\n",
       "             ('location', 26),\n",
       "             ('images',\n",
       "              [{'id': '5968c0f9-23d2-11e8-a6a3-ec086b02610b',\n",
       "                'frame_num': 1,\n",
       "                'datetime': '2013-10-04 13:31:53',\n",
       "                'file': '5968c0f9-23d2-11e8-a6a3-ec086b02610b.jpg',\n",
       "                'class': ['empty']}]),\n",
       "             ('rights_holder', 'Erin Boydston')])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cal_sequences[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n"
     ]
    }
   ],
   "source": [
    "sequences_schema_check.sequences_schema_check(cal_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1261/177122 [00:00<00:15, 11680.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 files found in directory at annotation_path\n",
      "Number of annotation entries found: 177122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 177122/177122 [00:09<00:00, 19244.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2773 boxes on 2615 images were in the annotation file(s). 212996 boxes skipped because they are not for the requested dataset\n",
      "\n",
      "Category counts for the bboxes:\n",
      "animal: 2\n",
      "person: 200\n",
      "vehicle: 2571\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "image_filename_to_bboxes = extract_annotations(annotation_path, dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset to which the sequences belong to: caltech. The bboxes should also be for this set.\n",
      "2615 images updated; 2615 images had their bbox overwritten; 241565 images not updated\n"
     ]
    }
   ],
   "source": [
    "cal_sequences, images_updated = add_annotations_to_sequences(cal_sequences, image_filename_to_bboxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/megadb_batches_9_10_11/caltech_w_batch_10_boxes.json', cal_sequences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## snapshotserengeti_private\n",
    "\n",
    "The images sent to get annotated all had the prefix `snapshotserengeti`.\n",
    "\n",
    "Need a separate `dataset_name` for the private set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/Source/temp_data/CameraTrap/batch_9_10_11/SS_1_11/SnapshotSerengeti_v2_0.json') as f:\n",
    "    all_ss = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6679039"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "6755090"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "61"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_ss['images'])\n",
    "len(all_ss['annotations'])\n",
    "len(all_ss['categories'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/databases_201908/snapshotserengeti/SnapshotSerengetiBboxes_20190903.json') as f:\n",
    "    all_ss_bbox = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading image DB...\n",
      "Number of items from the image DB: 6679039\n",
      "Number of images with more than 1 species: 75347 (1.13% of image DB)\n",
      "Loading bbox DB...\n",
      "Number of images added from bbox DB entries:  0\n",
      "Number of images amended:  82938\n",
      "Number of items in total:  6679039\n",
      "Number of images with more than one bounding box: 24905 (0.37288298511207973% of all entries)\n"
     ]
    }
   ],
   "source": [
    "ss_embedded = make_cct_embedded(image_db=all_ss, bbox_db=all_ss_bbox)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/snapshotserengeti_embedded.json', 'w') as f:\n",
    "    json.dump(ss_embedded, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Restarts here... `ss_embedded` is 3GB; `all_ss` is 5GB."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/cosmos/snapshotserengeti_embedded.json') as f:\n",
    "    ss_embedded = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 'S3/H03/H03_R11/S3_H03_R11_IMAG4457',\n",
       "  'file_name': 'S3/H03/H03_R11/S3_H03_R11_IMAG4457.JPG',\n",
       "  'frame_num': 3,\n",
       "  'seq_id': 'SER_S3#H03#11#1525',\n",
       "  'width': 2048,\n",
       "  'height': 1536,\n",
       "  'corrupt': False,\n",
       "  'location': 'H03',\n",
       "  'seq_num_frames': 3,\n",
       "  'datetime': '2011-12-27 10:31:05',\n",
       "  'annotations': {'species': ['empty'],\n",
       "   'season': 'S3',\n",
       "   'subject_id': 'ASG0009o9z',\n",
       "   'count': nan,\n",
       "   'standing': nan,\n",
       "   'resting': nan,\n",
       "   'moving': nan,\n",
       "   'interacting': nan,\n",
       "   'young_present': nan}}]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(ss_embedded, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine with S11 metadata\n",
    "with open('/Users/siyuyang/Source/temp_data/CameraTrap/batch_9_10_11/SS_1_11/SnapshotSerengetiS11.json') as f:\n",
    "    s11_cct = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading image DB...\n",
      "Number of items from the image DB: 499401\n",
      "Number of images with more than 1 species: 6727 (1.35% of image DB)\n",
      "No bbox DB provided.\n"
     ]
    }
   ],
   "source": [
    "s11_embedded = make_cct_embedded(image_db=s11_cct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 'SER_S11/G03/G03_R1/SER_S11_G03_R1_IMAG0750',\n",
       "  'file_name': 'SER_S11/G03/G03_R1/SER_S11_G03_R1_IMAG0750.JPG',\n",
       "  'frame_num': 3,\n",
       "  'seq_id': 'SER_S11#G03#1#334',\n",
       "  'width': 2560,\n",
       "  'height': 1920,\n",
       "  'corrupt': False,\n",
       "  'location': 'G03',\n",
       "  'seq_num_frames': 3,\n",
       "  'datetime': '2015-08-13 11:10:20',\n",
       "  'annotations': {'species': ['zebra'],\n",
       "   'season': 'SER_S11',\n",
       "   'subject_id': 21951579,\n",
       "   'count': '2',\n",
       "   'standing': 0.63,\n",
       "   'resting': 0.05,\n",
       "   'moving': 0.11,\n",
       "   'interacting': 0.0,\n",
       "   'young_present': 0.0}}]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(s11_embedded, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "ss_embedded.extend(s11_embeddededded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in ss_embedded:\n",
    "    if 'bbox' in e['annotations']:\n",
    "    # this DB already has the coordinates in relative coord\n",
    "        for b in e['annotations']['bbox']:\n",
    "            b['bbox_rel'] = b['bbox']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# how many images are there in each season?\n",
    "im_per_season = defaultdict(int)\n",
    "\n",
    "for e in ss_embedded:\n",
    "    season = e['id'].split('/')[0]\n",
    "    im_per_season[season] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {'S10': 685481,\n",
       "             'S2': 573200,\n",
       "             'S5': 827224,\n",
       "             'S6': 462846,\n",
       "             'S9': 982404,\n",
       "             'S8': 980256,\n",
       "             'S3': 392507,\n",
       "             'S7': 832153,\n",
       "             'S4': 531554,\n",
       "             'S1': 411414,\n",
       "             'SER_S11': 499401})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "im_per_season"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'snapshotserengeti'\n",
    "annotation_path = anno_batch_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 21701/7178440 [00:00<01:05, 108481.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset_name is set to snapshotserengeti. Please make sure this is correct!\n",
      "Putting 7178440 images into sequences...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7178440/7178440 [01:38<00:00, 73077.69it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sequences: 2659222\n",
      "Checking the location field...\n",
      "\n",
      "all_img_properties\n",
      "{'standing', 'bbox', 'class', 'subject_id', 'count', 'location', 'frame_num', 'moving', 'corrupt', 'id', 'young_present', 'datetime', 'file', 'resting', 'season', 'interacting'}\n",
      "\n",
      "img_level_properties\n",
      "{'file', 'id', 'frame_num', 'bbox'}\n",
      "\n",
      "image-level properties that really should be sequence-level\n",
      "{'standing', 'class', 'subject_id', 'count', 'location', 'moving', 'corrupt', 'young_present', 'datetime', 'resting', 'season', 'interacting'}\n",
      "\n",
      "! Sequence-level property corrupt with value False should be a dataset-level property. Removed from sequences.\n",
      "Finished processing sequences.\n",
      "Example sequence items:\n",
      "\n",
      "{'seq_id': 'SER_S10#C05#2#519', 'dataset': 'snapshotserengeti', 'images': [{'id': 'S10/C05/C05_R2/S10_C05_R2_IMAG1333', 'frame_num': 2, 'file': 'S10/C05/C05_R2/S10_C05_R2_IMAG1333.JPG'}, {'id': 'S10/C05/C05_R2/S10_C05_R2_IMAG1332', 'frame_num': 1, 'file': 'S10/C05/C05_R2/S10_C05_R2_IMAG1332.JPG'}, {'id': 'S10/C05/C05_R2/S10_C05_R2_IMAG1334', 'frame_num': 3, 'file': 'S10/C05/C05_R2/S10_C05_R2_IMAG1334.JPG'}], 'standing': nan, 'class': ['empty'], 'subject_id': 'ASG001xec7', 'count': nan, 'location': 'C05', 'moving': nan, 'young_present': nan, 'datetime': '2015-03-21 16:46:01', 'resting': nan, 'season': 'S10', 'interacting': nan}\n",
      "\n",
      "[{'seq_id': 'SER_S9#G04#2#598', 'dataset': 'snapshotserengeti', 'images': [{'id': 'S9/G04/G04_R2/S9_G04_R2_IMAG1422', 'frame_num': 3, 'file': 'S9/G04/G04_R2/S9_G04_R2_IMAG1422.JPG'}, {'id': 'S9/G04/G04_R2/S9_G04_R2_IMAG1421', 'frame_num': 2, 'file': 'S9/G04/G04_R2/S9_G04_R2_IMAG1421.JPG'}, {'id': 'S9/G04/G04_R2/S9_G04_R2_IMAG1420', 'frame_num': 1, 'file': 'S9/G04/G04_R2/S9_G04_R2_IMAG1420.JPG'}], 'standing': 0.6, 'class': ['wildebeest'], 'subject_id': 'ASG001i1h3', 'count': '8', 'location': 'G04', 'moving': 0.9, 'young_present': 0.0, 'datetime': '2014-10-26 17:23:37', 'resting': 0.0, 'season': 'S9', 'interacting': 0.0}]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ss_sequences = process_sequences(ss_embedded, dataset_name, deepcopy_docs=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/Source/temp_data/CameraTrap/batch_9_10_11/SS_1_11/ss_sequences.json', 'w') as f:\n",
    "    json.dump(ss_sequences, f)  # this is 1.6GB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n"
     ]
    }
   ],
   "source": [
    "sequences_schema_check.sequences_schema_check(ss_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 3191/177122 [00:00<00:05, 31903.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 files found in directory at annotation_path\n",
      "Number of annotation entries found: 177122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 177122/177122 [00:07<00:00, 22770.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4868 boxes on 3680 images were in the annotation file(s). 210901 boxes skipped because they are not for the requested dataset\n",
      "\n",
      "Category counts for the bboxes:\n",
      "animal: 840\n",
      "group: 10\n",
      "person: 2412\n",
      "vehicle: 1606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "image_filename_to_bboxes = extract_annotations(annotation_path, dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset to which the sequences belong to: snapshotserengeti. The bboxes should also be for this set.\n",
      "3680 images updated; 533 images had their bbox overwritten; 7174760 images not updated\n"
     ]
    }
   ],
   "source": [
    "ss_sequences, images_updated = add_annotations_to_sequences(ss_sequences, image_filename_to_bboxes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now `ss_sequences` contains all the past and batch 10 bboxes. Need to separate out the human ones. If any image belongs to the private set, move that sequence to the private set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "81176"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_im_w_box = 0\n",
    "for seq in ss_sequences:\n",
    "    for im in seq['images']:\n",
    "        if 'bbox' in im:\n",
    "            num_im_w_box += 1\n",
    "num_im_w_box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name_pri = 'snapshotserengeti_private'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/siyuyang/Source/temp_data/CameraTrap/batch_9_10_11/SS_1_11/list_humans.txt') as f:\n",
    "    list_human_ims = f.readlines()\n",
    "list_human_ims = [i.strip() for i in list_human_ims]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47015"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_human_ims = [i for i in list_human_ims if i.endswith('.JPG')]\n",
    "len(list_human_ims)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_human_ims = ['/'.join(i.split('/')[1:]) for i in list_human_ims]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47015"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "human_ims = set(list_human_ims)\n",
    "len(human_ims)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2659222/2659222 [01:34<00:00, 28131.78it/s]\n"
     ]
    }
   ],
   "source": [
    "ss_pri_seqs = []\n",
    "ss_pub_seqs = []\n",
    "\n",
    "for seq in tqdm(ss_sequences):\n",
    "    \n",
    "    has_hum_im = False\n",
    "    \n",
    "    for im in seq['images']:\n",
    "        if im['file'] in human_ims:\n",
    "            has_hum_im = True\n",
    "            break\n",
    "    if has_hum_im:\n",
    "        seq['dataset'] = dataset_name_pri\n",
    "        ss_pri_seqs.append(seq)\n",
    "    else:\n",
    "        ss_pub_seqs.append(seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17106"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "2642116"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ss_pri_seqs)\n",
    "len(ss_pub_seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json('/Users/siyuyang/Source/temp_data/CameraTrap/batch_9_10_11/SS_1_11/snapshotserengeti_private_w_batch_10_boxes.json', ss_pri_seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json('/Users/siyuyang/Source/temp_data/CameraTrap/batch_9_10_11/SS_1_11/snapshotserengeti_w_batch_10_boxes.json', ss_pub_seqs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make into MegaDB format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = idfg_embedded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'idfg'\n",
    "partial_mega_db_path = ''\n",
    "annotation_path = anno_batch_10\n",
    "mega_db_with_bbox = '/Users/siyuyang/OneDrive - Microsoft/AI4Earth/CameraTrap/Databases/megadb_batches_9_10_11/idfg_w_batch_10_boxes_fixed.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset_name is set to idfg. Please make sure this is correct!\n",
      "Making a deep copy of docs...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 15408/693886 [00:00<00:08, 77030.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Putting 693886 images into sequences...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 693886/693886 [00:08<00:00, 80554.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sequences: 693886\n",
      "Checking the location field...\n",
      "Checking which fields in a CCT image entry are sequence-level...\n",
      "\n",
      "all_img_properties\n",
      "{'bbox', 'class', 'file', 'dataset', 'datetime', 'location', 'image_id'}\n",
      "\n",
      "img_level_properties\n",
      "{'bbox', 'file', 'image_id', 'datetime'}\n",
      "\n",
      "image-level properties that really should be sequence-level\n",
      "{'class', 'dataset', 'location'}\n",
      "\n",
      "Finished processing sequences.\n",
      "Example sequence items:\n",
      "\n",
      "{'seq_id': 'dummy_169821ab557440368b9bc9e03ca2ff46', 'dataset': 'idfg', 'images': [{'image_id': 'ClearCreek_mustelids~Summer2015~FS-034~RCNX2002', 'datetime': '8/25/15 7:41:12', 'file': 'ClearCreek_mustelids/Summer2015/FS-034/RCNX2002.JPG'}], 'class': ['cattle'], 'location': 'ClearCreek_mustelids+FS-034'}\n",
      "\n",
      "[{'seq_id': 'dummy_4bf420bdc7f64da0bc68502672051d98', 'dataset': 'idfg', 'images': [{'image_id': 'St.Joe_elk~AM164~Trip 1~100RECNX~RCNX5004', 'datetime': '18-Mar-2016 12:16:56', 'file': 'St.Joe_elk/AM164/Trip 1/100RECNX/RCNX5004.JPG'}], 'class': ['empty'], 'location': 'St_Joe_elk+AM164'}]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sequences = process_sequences(docs, dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Verified that the sequence items meet requirements not captured by the schema.\n",
      "Verified that the sequence items conform to the schema.\n"
     ]
    }
   ],
   "source": [
    "sequences_schema_check.sequences_schema_check(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write_json(partial_mega_db_path, sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add bounding boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▎         | 6361/177122 [00:00<00:05, 31793.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19 files found in directory at annotation_path\n",
      "Number of annotation entries found: 177122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 177122/177122 [00:06<00:00, 29340.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43 boxes on 29 images were in the annotation file(s). 215726 boxes skipped because they are not for the requested dataset\n",
      "\n",
      "Category counts for the bboxes:\n",
      "person: 15\n",
      "vehicle: 28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "image_filename_to_bboxes = extract_annotations(annotation_path, dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset to which the sequences belong to: idfg. The bboxes should also be for this set.\n",
      "29 images updated; 0 images had their bbox overwritten; 693857 images not updated\n"
     ]
    }
   ],
   "source": [
    "sequences, images_updated = add_annotations_to_sequences(sequences, image_filename_to_bboxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[OrderedDict([('dataset', 'idfg'),\n",
       "              ('seq_id', 'dummy_0f3eef05b402408787513e1268d17d06'),\n",
       "              ('location', 'St_Joe_elk+AM142'),\n",
       "              ('images',\n",
       "               [{'image_id': 'St.Joe_elk~AM142~Trip 1~100RECNX~RCNX0374',\n",
       "                 'datetime': '15-Feb-2016 02:25:05',\n",
       "                 'file': 'St.Joe_elk/AM142/Trip 1/100RECNX/RCNX0374.JPG'}]),\n",
       "              ('class', ['empty'])]),\n",
       " OrderedDict([('dataset', 'idfg'),\n",
       "              ('seq_id', 'dummy_00bc06d550144816a5378bc153cf5373'),\n",
       "              ('location', 'ClearCreek_mustelids+FS-045'),\n",
       "              ('images',\n",
       "               [{'image_id': 'ClearCreek_mustelids~Winter2015-16~FS-045-P~RCNX0932',\n",
       "                 'datetime': '11/26/15 16:28:56',\n",
       "                 'file': 'ClearCreek_mustelids/Winter2015-16/FS-045-P/RCNX0932.JPG',\n",
       "                 'bbox': [{'category': 'animal',\n",
       "                   'bbox': [0.0587, 0.663, 0.197, 0.191]}]}]),\n",
       "              ('class', ['deer'])])]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample(sequences, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_json(mega_db_with_bbox, sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:cameratraps] *",
   "language": "python",
   "name": "conda-env-cameratraps-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
